package com.chance.cc.crawler.development.bootstrap.iqiyi;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerJob;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import com.chance.cc.crawler.development.utils.RedisReader;
import com.chance.cc.crawler.meta.core.bean.CrawlerMetaConstant;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Set;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRecordFilter.key;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

public class IQYArticleSimpleStart {

    public static final String domain = "iqiyi";//
    public static final String site = "searchKw";//
    public static final String siteBiz = "video";//
    public static final RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 4);//
    public static final String iqyArticleUrlFormat = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/note/%s/single_feed";//

    private static Proxy proxy = new Proxy();

    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    HttpConfig httpConfig = HttpConfig.me(domain);

    public static void main(String[] args) throws Exception {
        send2Redis("文件地址");//
    }

    public static void send2Redis(String filePath) throws Exception {
        List<String> localUrls = IOUtils.readLines(new FileInputStream(filePath), StandardCharsets.UTF_8);
        for (String localUrl : localUrls) {
            try {
                String articleId = localUrl.split("item/")[1];
                String articleUrl = String.format(iqyArticleUrlFormat, articleId);//
                CrawlerRequestRecord requestRecord = genCrawlerArticleRecord(articleUrl);

                String requestKey = requestRecord.getRecordKey();
                //确保key值唯一
                String shaKey = DigestUtils.sha1Hex(System.currentTimeMillis() + requestKey);
                redisReader.mapPush(StringUtils.joinWith("-", "crawler", domain, site, "simple", "queue_map"), shaKey, JSON.toJSONString(requestRecord));
                redisReader.listPush(StringUtils.joinWith("-", "crawler", domain, site, "simple", "queue_list"), JSON.toJSONString(requestRecord));
//                System.out.println("add key : " + shaKey + " success!");

            } catch (Exception e) {
                System.out.println("*----------------*" + localUrl);
                e.printStackTrace();
            }
        }
    }

    private static CrawlerRequestRecord genCrawlerArticleRecord(String articleUrl) {
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPageItem)
                .domain(domain)
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 30, null))
                .httpUrl(articleUrl)
                .recordKey(articleUrl)//
                .releaseTime(System.currentTimeMillis())
                .notFilterRecord()
//                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                .needParsed(true)
                .needWashed(true)
                .proxy(proxy)
                .build();
        requestRecord.setDownload(false);//
        requestRecord.setSkipPipeline(true);//
        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);


//        //添加评论去重信息
        CrawlerRecord filterCrawlerRecord = new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
        filterCrawlerRecord.addFilterInfo(FilterUtils.redisFilterKeyInfo(StringUtils.joinWith("-", filter, domain, site, "comment")));
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 365 * 10, null));
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_filter_record", JSON.toJSONString(filterCrawlerRecord));
        requestRecord.getHttpRequest().addExtra("searchKwSourceUrl", "https://so.iqiyi.com/so/q_%s_ctg__t_0_page_1_p_1_qc_0_rd__site_iqiyi_m_4_bitrate__af_0");//
//        requestRecord.tagsCreator().scheduleTags().getCategoryTag().addLabelTag(CrawlerEnum.CrawlerDataType.comment.enumVal());
        return requestRecord;
    }


}
